﻿#pragma warning disable 0168
using System;
using System.Collections.Generic;
using System.Linq;
using Celeriq.Utilities;
using System.IO;
using Celeriq.Common;
using System.Diagnostics;
using System.Xml;
using Celeriq.Server.Interfaces;

namespace Celeriq.RepositoryAPI
{
    internal class Repository
    {
        #region Class Members

        private const int LOCK_TIMEOUT = 500; //For read/write locks
        private const int THREAD_THRESHOLD = 100000; //Number of records needed to use threading
        private const int THREAD_CORE_COUNT = 3; //Number of processors needed to use threading
        private const bool CACHE_ON = true;
        private const int MAX_PROFILE_COUNT = 1000;

        private System.Threading.ReaderWriterLockSlim _locker = new System.Threading.ReaderWriterLockSlim();
        private RepositoryDefinition _repositoryDefinition = null;
        private QueryLogger _queryLogger;

        private readonly object _globalLock = new object();
        private List<DataItemExtension> _list = null;
        private List<DimensionItem> _dimensionList = null;
        private List<FileCacheHelper<RefinementItem>> _dDimensionCache = null;
        private int _pkindex = -1; //the index in the data array of the primary key

        private Dictionary<long, List<DataItemExtension>> _dimensionMappedItemCache = null;

        private string _cacheFolder = string.Empty;
        private string _cacheFile = string.Empty;
        private long _maxRecordIndex = 1;
        private int _coreCount = 0;

        private HashSet<int> _pkList = new HashSet<int>();
        private FileCacheHelper<long> _deletedCache = null;
        private HashSet<long> _deletedList = new HashSet<long>();
        private SequencedHashTable<int, DataQueryResults> _resultsCache = null;
        private FileCacheHelper<DataItemExtension> _fileCache = null;
        private CacheControl _cacheControl = null;
        private Celeriq.Server.Interfaces.ServiceStartup _startup = null;
        private long _versionHash = 0;
        private DateTime _lastAccess = DateTime.MinValue;
        private bool _isLoaded = false;
        private KeyPair _masterKeys = null;

        //Determines if the cache file should be rebuilt because the item list changed
        private bool _rebuildCache = false;
        private DateTime _lastChange = DateTime.MinValue;
        private System.Timers.Timer _timer = null;
        private long _memorySize = 0;
        private Celeriq.Client.Interfaces.ISystemCore _system = null;

        private List<ProfileItem> _profileList = new List<ProfileItem>();

        //Diagnostic data
        private DataItem _lastInserted = null;
        private bool _memoryDirty = true;

        #endregion

        #region Constructors

        public Repository(Celeriq.Server.Interfaces.ServiceStartup startup, KeyPair masterKeys, Celeriq.Client.Interfaces.ISystemCore system)
        {
            _system = system;
            _masterKeys = masterKeys;
            _startup = startup;
            _repositoryDefinition = startup.RepositoryDefinition;
            _cacheFolder = Path.Combine(startup.RepositoryFolder, startup.RepositoryDefinition.ID.ToString());

            _timer = new System.Timers.Timer(5000);
            _timer.Elapsed += _timer_Elapsed;
            _timer.Start();

            //Call this to reload cache file
            GetItemCountFromCache();
        }

        private bool _isReloading = false;

        private void ReloadMe(Celeriq.Server.Interfaces.ServiceStartup startup)
        {
            try
            {
                if (_isReloading) return;
                _isReloading = true;

                var timer = new CQTimer();

                //Create the data folder
                if (!Directory.Exists(_cacheFolder))
                {
                    Directory.CreateDirectory(_cacheFolder);
                }

                _queryLogger = new QueryLogger(_cacheFolder);
                _repositoryDefinition = startup.RepositoryDefinition;
                _versionHash = _repositoryDefinition.VersionHash;
                _pkindex = _repositoryDefinition.FieldList.IndexOf(_repositoryDefinition.PrimaryKey);

                //Create the master list
                _list = new List<DataItemExtension>();
                _coreCount = Environment.ProcessorCount;

                _cacheControl = new CacheControl();

                //Load the dimension files
                _dDimensionCache = new List<FileCacheHelper<RefinementItem>>();
                var index = 0;
                foreach (var d in this._repositoryDefinition.DimensionList)
                {
                    var f = new FileCacheHelper<RefinementItem>(Path.Combine(_cacheFolder, "d1" + index.ToString("000000") + ".data"));
                    _dDimensionCache.Add(f);
                    index++;
                }

                _deletedCache = new FileCacheHelper<long>(Path.Combine(_cacheFolder, "deleted.data"));
                LoadDeletedList();

                //Create the dimension lists
                _dimensionList = new List<DimensionItem>();
                index = 0;
                foreach (var d in this._repositoryDefinition.DimensionList)
                {
                    _dimensionList.Add(new DimensionItem() { Name = d.Name, DIdx = DimensionDefinition.DGROUP + index, NumericBreak = d.NumericBreak });
                    index++;
                }

                //Initialize the dimensions from file
                var timer2 = new Stopwatch();
                timer2.Start();
                index = 0;
                foreach (var d in this._repositoryDefinition.DimensionList)
                {
                    InitializeDimension(_dimensionList[index], _dDimensionCache[index]);
                    index++;
                }
                timer2.Stop();
                Logger.LogInfo("InitializeDimension All: (" + timer2.ElapsedMilliseconds.ToString("###,###,###,##0") + " ms");

                //The data files
                _cacheFile = Path.Combine(_cacheFolder, "repository.data");

                _fileCache = new FileCacheHelper<DataItemExtension>(_cacheFile);

                //Load dimension/item mapping cache
                _dimensionMappedItemCache = new Dictionary<long, List<DataItemExtension>>();
                foreach (var d in _dimensionList)
                {
                    foreach (var r in d.RefinementList)
                    {
                        _dimensionMappedItemCache.Add(r.DVIdx, new List<DataItemExtension>());
                    }
                }

                LoadData();

                _system.NotifyLoad(_repositoryDefinition.ID);

                timer.Stop();
                Logger.LogInfo("Repository Data Loaded: " + startup.RepositoryDefinition.ID.ToString() + " (" + timer.Elapsed.ToString("###,###,###,##0") + " ms | " + _list.Count.ToString("###,###,###,##0") + " items | " + (_list.Count / (timer.Elapsed / 1000.0)).ToString("###,###,###,##0") + " items/s)");
                AddProfileItem(ActionConstants.Load, timer);
            }
            catch (Exception ex)
            {
                Logger.LogError("Repository Loading Error\n" + ex.ToString());
                throw;
            }
            finally
            {
                _isReloading = false;
            }
        }

        #endregion

        private delegate void LogActionDelegate(DataQuery query, int elapsed, int count, bool fromcache);

        #region Properties

        public DateTime LastAccess
        {
            get { return _lastAccess; }
        }

        public bool IsLoaded
        {
            get { return _isLoaded; }
        }

        #endregion

        #region Unload

        public void UnloadData()
        {
            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    if (!_isLoaded) return;
                    var timer = new CQTimer();

                    //Ensure all is saved
                    SaveData();

                    //Clear all data from memory
                    if (_list != null) _list.Clear();
                    _list = null;
                    if (_dimensionList != null) _dimensionList.Clear();
                    _dimensionList = null;
                    if (_dDimensionCache != null)
                        _dDimensionCache.ForEach(x => (x as IDisposable).Dispose());
                    _dDimensionCache = null;
                    _pkindex = -1;
                    if (_dimensionMappedItemCache != null) _dimensionMappedItemCache.Clear();
                    _dimensionMappedItemCache = null;
                    _maxRecordIndex = 1;
                    _coreCount = 0;
                    if (_pkList != null) _pkList.Clear();
                    _pkList = new HashSet<int>();
                    if (_deletedCache != null)
                        (_deletedCache as IDisposable).Dispose();
                    _deletedCache = null;
                    _deletedList = new HashSet<long>();
                    if (_resultsCache != null) _resultsCache.Clear();
                    _resultsCache = null;
                    if (_fileCache != null)
                        (_fileCache as IDisposable).Dispose();
                    _fileCache = null;
                    _cacheControl = null;
                    _versionHash = 0;

                    //Mark this repository as unloaded
                    _lastAccess = DateTime.MinValue;
                    _isLoaded = false;

                    AddProfileItem(ActionConstants.Unload, timer);
                    Logger.LogInfo("Repository UnloadData: " + _startup.RepositoryDefinition.ID.ToString());

                    _system.NotifyUnload(_repositoryDefinition.ID);

                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    throw;
                }
            }
        }

        #endregion

        #region UpdateIndex

        public void UpdateIndexList(IEnumerable<DataItem> list, UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    if (!this.IsLoaded)
                        ReloadMe(_startup);

                    if (list == null) return;
                    if (list.Count() == 0) return;

                    Console.WriteLine("UpdateIndexList: " + _repositoryDefinition.ID);
                    var timer = new Stopwatch();
                    timer.Start();

                    //Add the actual items to the master list
                    foreach (var item in list)
                    {
                        var timerInner = new CQTimer();

                        _lastAccess = DateTime.Now;
                        var newItem = new DataItemExtension(item, _repositoryDefinition);
                        if (!IsItemValid(newItem))
                            throw new Exception("The item is not valid!");

                        //Check to see if this item exists and if so remove it
                        this.DeleteItem((int)newItem.ItemArray[_pkindex]);

                        newItem.__RecordIndex = _maxRecordIndex;
                        _maxRecordIndex++;
                        _list.Add(newItem);
                        _memoryDirty = true;
                        newItem.__RecordIndex = newItem.__RecordIndex;

                        _dimensionList.ForEach(x => newItem.DimensionSingularValueArray.Add(null));
                        ProcessDimensions(newItem);

                        _fileCache.WriteItem(newItem);
                        _lastInserted = newItem;

                        _pkList.Add((int)newItem.ItemArray[_pkindex]);

                        //Clear cache
                        _resultsCache.Clear();
                        _rebuildCache = true;
                        _lastChange = DateTime.Now;

                        AddProfileItem(ActionConstants.Update, timerInner);
                    }

                    timer.Stop();
                    _system.LogRepositoryPerf(new RepositorySummmaryStats
                                              {
                                                  ActionType = RepositoryActionConstants.Update,
                                                  Elapsed = (int)timer.ElapsedMilliseconds,
                                                  RepositoryId = _repositoryDefinition.ID,
                                                  QueryCount = 1,
                                                  //QueryCount = list.Count(),
                                              });
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    throw;
                }
            }
        }

        private string CacheFile
        {
            get { return Path.Combine(_cacheFolder, "repository.cache"); }
        }

        private long GetItemCountFromCache()
        {
            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    if (File.Exists(this.CacheFile))
                    {
                        var document = new XmlDocument();
                        document.Load(this.CacheFile);
                        _memorySize = XmlHelper.GetAttribute(document.DocumentElement, "memorysize", 0);
                        return XmlHelper.GetAttribute(document.DocumentElement, "itemcount", 0);
                    }
                    else
                    {
                        return 0;
                    }
                }
                catch (Exception ex)
                {
                    Logger.LogError("GetItemCountFromCache Error\n" + ex.ToString());
                    return 0;
                }
            }
        }

        private void WriteCacheFile()
        {
            try
            {
                _lastChange = DateTime.Now;
                _rebuildCache = false;
                _memorySize = ObjectHelper.SizeOf(_list);
                File.WriteAllText(this.CacheFile, "<repository itemcount=\"" + _pkList.Count + "\" memorysize=\"" + _memorySize + "\"></repository>");
            }
            catch (Exception ex)
            {
                Logger.LogError("WriteCacheFile Error\n" + ex.ToString());
            }
        }

        #endregion

        #region DeleteData

        public void DeleteData(IEnumerable<DataItem> list, UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    if (!this.IsLoaded)
                    {
                        ReloadMe(_startup);
                    }

                    var timer = new Stopwatch();
                    timer.Start();

                    //Console.WriteLine("DeleteData: " + _repositoryDefinition.ID);
                    list.ToList().ForEach(x => DeleteItem((int)x.ItemArray[_pkindex]));
                    _rebuildCache = true;
                    _lastChange = DateTime.Now;

                    timer.Stop();
                    _system.LogRepositoryPerf(new RepositorySummmaryStats
                    {
                        ActionType = RepositoryActionConstants.Delete,
                        Elapsed = (int)timer.ElapsedMilliseconds,
                        RepositoryId = _repositoryDefinition.ID,
                        QueryCount = 1,
                        //QueryCount = list.Count(),
                    });

                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        private void DeleteItem(int primaryKey)
        {
            try
            {
                _lastAccess = DateTime.Now;
                //Check if this item is in the list
                if (!_pkList.Contains(primaryKey))
                    return;

                var timer = new CQTimer();
                //If this item is in the list then actually find it and remove it
                var existingItem = _list.AsParallel().FirstOrDefault(x => (int)x.ItemArray[_pkindex] == primaryKey);
                if (existingItem != null)
                {
                    _list.Remove(existingItem);
                    _deletedList.Add(existingItem.__RecordIndex);
                    _deletedCache.WriteItem(existingItem.__RecordIndex);

                    // remove the item from the _dimensions
                    foreach (var val in existingItem.DimensionValueArray)
                    {
                        _dimensionMappedItemCache[val].Remove(existingItem);
                    }
                }

                //Clear cache
                _resultsCache.Clear();

                AddProfileItem(ActionConstants.Delete, timer);
            }
            catch (Exception ex)
            {
                Logger.LogError(ex);
                throw;
            }
        }

        #endregion

        #region Query

        public DataQueryResults Query(DataQuery query)
        {
            if (!IsValidCredentials(query.Credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireReaderLock(_locker))
            {
                try
                {
                    if (!this.IsLoaded)
                    {
                        ReloadMe(_startup);
                    }

                    _lastAccess = DateTime.Now;
                    var newResults = new DataQueryResults();
                    newResults.DimensionList = new List<DimensionItem>();
                    newResults.RecordList = new List<DataItem>();
                    newResults.Query = query;
                    newResults.VersionHash = _versionHash;

                    if (_list.Count == 0)
                    {
                        return newResults;
                    }

                    //Clear the cache if need be
                    if (!string.IsNullOrEmpty(query.NonParsedFieldList["clearcache"]))
                        ClearCache();

                    var qHash = query.GetHashCode();
                    if (CACHE_ON && _resultsCache.ContainsKey(qHash))
                    {
                        var timerCache = new CQTimer();
                        var cacheItem = _resultsCache[qHash];
                        try
                        {
                            if (DateTime.Now.Subtract(cacheItem.QueryTime).TotalSeconds < _startup.CacheLength)
                            {
                                _queryLogger.Log(query, 0, cacheItem.TotalRecordCount, true);
                                return cacheItem;
                            }
                        }
                        catch (Exception ex)
                        {
                            Logger.LogError(ex);
                            throw;
                        }
                        finally
                        {
                            AddProfileItem(ActionConstants.Query, timerCache, query.ToString(), cacheItem.TotalRecordCount);
                            _system.LogRepositoryPerf(new RepositorySummmaryStats
                            {
                                ActionType = RepositoryActionConstants.Query,
                                Elapsed = timerCache.Elapsed,
                                RepositoryId = _repositoryDefinition.ID,
                                QueryCount = 1,
                                //QueryCount = cacheItem.RecordList.Count,
                            });
                        }
                    }

                    var timer = new CQTimer();
                    var _timeList = new List<string>();
                    var myWatch2 = new Stopwatch();

                    IEnumerable<DataItemExtension> queriedList = null;

                    //Apply dimension indexes first
                    if (query.DimensionValueList != null && query.DimensionValueList.Any())
                    {
                        var prepList = new Dictionary<DimensionItem, List<long>>();
                        foreach (var dvidx in query.DimensionValueList)
                        {
                            var dItem = _dimensionList.GetDimensionByDVIdx(dvidx);
                            if (dItem != null)
                            {
                                if (!prepList.ContainsKey(dItem)) prepList.Add(dItem, new List<long>());
                                prepList[dItem].Add(dvidx);
                            }
                        }

                        foreach (var dItem in prepList.Keys)
                        {
                            var dDef = _repositoryDefinition.DimensionList.First(x => x.Name == dItem.Name);

                            //Union all values within a dimension
                            IEnumerable<DataItemExtension> tempDList = null;
                            foreach (var dvidx in prepList[dItem])
                            {
                                if (tempDList == null)
                                {
                                    tempDList = _dimensionMappedItemCache[dvidx];
                                }
                                else
                                {
                                    //List dimension refinements always INTERSECT values, all other Union
                                    if (dDef.DataType == RepositoryDefinition.DataTypeConstants.List)
                                        tempDList = tempDList.Intersect(_dimensionMappedItemCache[dvidx]);
                                    else
                                        tempDList = tempDList.Union(_dimensionMappedItemCache[dvidx]);
                                }
                            }

                            //Intersect group between dimensions
                            if (queriedList == null) queriedList = tempDList;
                            else
                            {
                                queriedList = queriedList.Intersect(tempDList);
                            }
                        }


                        //var filterDList = new List<DimensionItem>();
                        //foreach (var dvidx in query.DimensionValueList)
                        //{
                        //	var dItem = _dimensionList.GetDimensionByDVIdx(dvidx);
                        //	if (dItem != null)
                        //	{
                        //		var dDef = _repositoryDefinition.DimensionList.First(x => x.Name == dItem.Name);
                        //		if (_dimensionMappedItemCache.ContainsKey(dvidx))
                        //		{
                        //			if (queriedList == null) queriedList = _dimensionMappedItemCache[dvidx];
                        //			else
                        //			{
                        //				if (dDef.DataType == RepositoryDefinition.DataTypeConstants.List)
                        //				{
                        //					//List dimension refinements always INTERSECT values
                        //					queriedList = queriedList.Intersect(_dimensionMappedItemCache[dvidx]);
                        //				}
                        //				else //All other dimension types
                        //				{
                        //					//Multiple refinements in a dimension should UNION values
                        //					//Refinements across dimensions should INTERSECT
                        //					if (filterDList.Contains(dItem))
                        //						queriedList = queriedList.Union(_dimensionMappedItemCache[dvidx]);
                        //					else
                        //						queriedList = queriedList.Intersect(_dimensionMappedItemCache[dvidx]);
                        //				}
                        //			}
                        //		}

                        //		if (!filterDList.Contains(dItem))
                        //			filterDList.Add(dItem);
                        //	}
                        //}
                    }

                    if (queriedList == null) queriedList = _list;

                    #region Filters

                    if (query.FieldFilters != null)
                    {
                        //Verify there is only one Geo filter
                        if (query.FieldFilters.Count(x => x is GeoCodeFieldFilter) > 1)
                        {
                            throw new Exception("Multiple geo location filters cannot be specified!");
                        }

                        foreach (var filter in query.FieldFilters)
                        {
                            var ff = ((ICloneable)filter).Clone() as IFieldFilter;
                            if (ff == null) throw new Exception("Object cannot be null!");
                            var field = _repositoryDefinition.FieldList.First(x => x.Name == ff.Name);
                            if (field == null) throw new Exception("Object cannot be null!");
                            var fieldIndex = _repositoryDefinition.FieldList.IndexOf(field);

                            #region GeoCode

                            if (field.DataType == RepositoryDefinition.DataTypeConstants.GeoCode)
                            {
                                var geo = (GeoCodeFieldFilter)ff;
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.LessThan:
                                    case ComparisonConstants.LessThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && GeoHelper.InRadius(((GeoCode)x.ItemArray[fieldIndex]).Latitude, ((GeoCode)x.ItemArray[fieldIndex]).Longitude, geo.Latitude, geo.Longitude, geo.Radius));
                                        break;
                                    case ComparisonConstants.GreaterThan:
                                    case ComparisonConstants.GreaterThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && GeoHelper.OutRadius(((GeoCode)x.ItemArray[fieldIndex]).Latitude, ((GeoCode)x.ItemArray[fieldIndex]).Longitude, geo.Latitude, geo.Longitude, geo.Radius));
                                        break;
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && GeoHelper.Calc(((GeoCode)x.ItemArray[fieldIndex]).Latitude, ((GeoCode)x.ItemArray[fieldIndex]).Longitude, geo.Latitude, geo.Longitude) == geo.Radius);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && GeoHelper.Calc(((GeoCode)x.ItemArray[fieldIndex]).Latitude, ((GeoCode)x.ItemArray[fieldIndex]).Longitude, geo.Latitude, geo.Longitude) != geo.Radius);
                                        break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }
                            #endregion
                            #region Bool

                            else if (field.DataType == RepositoryDefinition.DataTypeConstants.Bool)
                            {
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (bool)x.ItemArray[fieldIndex] == (bool)ff.Value);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (bool)x.ItemArray[fieldIndex] != (bool)ff.Value);
                                        break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }
                            #endregion
                            #region DateTime

                            else if (field.DataType == RepositoryDefinition.DataTypeConstants.DateTime)
                            {
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.LessThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] < (DateTime)ff.Value);
                                        break;
                                    case ComparisonConstants.LessThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] <= (DateTime)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] > (DateTime)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] >= (DateTime)ff.Value);
                                        //queriedList = queriedList.Intersect(_list.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] >= (DateTime)ff.Value));
                                        //queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] >= (DateTime)ff.Value).ToList();
                                        break;
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] == (DateTime)ff.Value);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (DateTime)x.ItemArray[fieldIndex] != (DateTime)ff.Value);
                                        break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }
                            #endregion
                            #region Float

                            else if (field.DataType == RepositoryDefinition.DataTypeConstants.Float)
                            {
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.LessThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] < (double)ff.Value);
                                        break;
                                    case ComparisonConstants.LessThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] <= (double)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] > (double)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] >= (double)ff.Value);
                                        break;
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] == (double)ff.Value);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (double)x.ItemArray[fieldIndex] != (double)ff.Value);
                                        break;
                                    case ComparisonConstants.Between:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && ff.Value2 != null && (double)x.ItemArray[fieldIndex] >= (double)ff.Value && (double)x.ItemArray[fieldIndex] <= (double)ff.Value2);
                                        break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }
                            #endregion
                            #region Int

                            else if (field.DataType == RepositoryDefinition.DataTypeConstants.Int)
                            {
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.LessThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] < (int)ff.Value);
                                        break;
                                    case ComparisonConstants.LessThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] <= (int)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThan:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] > (int)ff.Value);
                                        break;
                                    case ComparisonConstants.GreaterThanOrEq:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] >= (int)ff.Value);
                                        break;
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] == (int)ff.Value);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && (int)x.ItemArray[fieldIndex] != (int)ff.Value);
                                        break;
                                    case ComparisonConstants.Between:
                                        queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && ff.Value2 != null && (int)x.ItemArray[fieldIndex] >= (int)ff.Value && (int)x.ItemArray[fieldIndex] <= (int)ff.Value2);
                                        break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }
                            #endregion
                            #region String

                            else if (field.DataType == RepositoryDefinition.DataTypeConstants.String)
                            {
                                switch (ff.Comparer)
                                {
                                    case ComparisonConstants.LessThan:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) < 0);
                                        break;
                                    case ComparisonConstants.LessThanOrEq:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) <= 0);
                                        break;
                                    case ComparisonConstants.GreaterThan:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) > 0);
                                        break;
                                    case ComparisonConstants.GreaterThanOrEq:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) >= 0);
                                        break;
                                    case ComparisonConstants.Equals:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) == 0);
                                        break;
                                    case ComparisonConstants.NotEqual:
                                        queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) != 0);
                                        //queriedList = queriedList.Intersect(_list.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) != 0));
                                        //queriedList = queriedList.Where(x => string.Compare((string)x.ItemArray[fieldIndex], (string)ff.Value) != 0).ToList();
                                        break;
                                    //case ComparisonConstants.Between:
                                    //	queriedList = queriedList.Where(x => x.ItemArray[fieldIndex] != null && ff.Value != null && ff.Value2 != null && (string)x.ItemArray[fieldIndex] >= (string)ff.Value && (string)x.ItemArray[fieldIndex] <= (string)ff.Value2);
                                    //	break;
                                    default:
                                        throw new Exception("This operation is not supported!");
                                }
                            }

                            #endregion
                        }
                    }

                    #endregion

                    #region Then do text search

                    var keywordSearchablelist = _repositoryDefinition.FieldList.Where(x => x.AllowTextSearch && x.DataType == RepositoryDefinition.DataTypeConstants.String).ToList();
                    if (!string.IsNullOrEmpty(query.Keyword) && keywordSearchablelist.Count > 0)
                    {
                        var keywordList = query.Keyword.Split(new char[] { ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
                        System.Linq.Expressions.Expression<Func<DataItemExtension, bool>> w = x => false;
                        foreach (var field in keywordSearchablelist)
                        {
                            var index = _repositoryDefinition.FieldList.IndexOf(field);
                            foreach (var kw in keywordList)
                            {
                                w = w.Or(x =>
                                         x.ItemArray[index] != null &&
                                         ((string)x.ItemArray[index]).Contains(kw, StringComparison.OrdinalIgnoreCase)
                                    );
                            }
                        }
                        queriedList = queriedList.Where(w.Compile());
                    }

                    #endregion

                    //Determine SortBy lambda
                    if (query.FieldSorts != null && query.FieldSorts.Any())
                    {
                        OrderedParallelQuery<DataItemExtension> tempOrder = null;
                        foreach (var sf in query.FieldSorts)
                        {
                            var field = _repositoryDefinition.FieldList.First(x => x.Name == sf.Name);
                            var index = _repositoryDefinition.FieldList.IndexOf(field);
                            if (sf.SortDirection == SortDirectionConstants.Asc)
                            {
                                if (tempOrder == null) tempOrder = queriedList.AsParallel().OrderBy(x => x.ItemArray[index]);
                                else tempOrder = tempOrder.ThenBy(x => x.ItemArray[index]);
                            }
                            else if (sf.SortDirection == SortDirectionConstants.Desc)
                            {
                                if (tempOrder == null) tempOrder = queriedList.AsParallel().OrderByDescending(x => x.ItemArray[index]);
                                else tempOrder = tempOrder.ThenByDescending(x => x.ItemArray[index]);
                            }
                        }
                        queriedList = tempOrder.ThenBy(x => x.__RecordIndex);
                    }
                    else
                    {
                        //Tack on recordindex sort for consistency
                        queriedList = queriedList.AsParallel().OrderBy(x => x.__RecordIndex);
                    }

                    myWatch2.Stop();
                    myWatch2.Reset();
                    myWatch2.Start();

                    //var queriedList2 = queriedList.ToList();

                    myWatch2.Stop();
                    _timeList.Add(string.Empty + myWatch2.ElapsedMilliseconds);
                    myWatch2.Reset();
                    myWatch2.Start();

                    //Convert to list
                    var queriedListRealized = queriedList.AsParallel().ToList();
                    queriedList = null;

                    myWatch2.Stop();
                    _timeList.Add(string.Empty + myWatch2.ElapsedMilliseconds);
                    myWatch2.Reset();
                    myWatch2.Start();

                    var isZeroResult = false;
                    var skipCount = (query.PageOffset - 1) * query.RecordsPerPage;
                    var takeCount = query.RecordsPerPage;
                    if (skipCount >= queriedListRealized.Count)
                    {
                        isZeroResult = true;
                    }
                    else
                    {
                        if (skipCount + takeCount > queriedListRealized.Count)
                            takeCount = queriedListRealized.Count - skipCount;
                    }

                    if (query.RecordsPerPage < 1) isZeroResult = true;

                    //var newResults = new DataQueryResults();
                    //newResults.DimensionList = new List<DimensionItem>();
                    //newResults.RecordList = new List<DataItem>();
                    if (!isZeroResult)
                    {
                        queriedListRealized.Skip(skipCount).Take(takeCount).ToList().ForEach(x => newResults.RecordList.Add(x.ToSerialized()));
                    }
                    newResults.TotalRecordCount = queriedListRealized.Count;

                    //Process the distance for geo-located filters
                    var geoFilter = query.FieldFilters.FirstOrDefault(x => x is GeoCodeFieldFilter) as GeoCodeFieldFilter;
                    if (geoFilter != null)
                    {
                        var fieldDef = _repositoryDefinition.FieldList.FirstOrDefault(x => x.Name == geoFilter.Name);
                        if (fieldDef != null)
                        {
                            var fieldIndex = _repositoryDefinition.FieldList.IndexOf(fieldDef);
                            if (fieldIndex != -1)
                            {
                                foreach (var item in newResults.RecordList)
                                {
                                    var geoField = item.ItemArray[fieldIndex] as GeoCode;
                                    if (geoField == null) throw new Exception("Object cannot be null!");
                                    geoField.Distance = GeoHelper.Calc(geoFilter.Latitude, geoFilter.Longitude, geoField.Latitude, geoField.Longitude);
                                }
                            }
                        }
                    }

                    myWatch2.Stop();
                    _timeList.Add(string.Empty + myWatch2.ElapsedMilliseconds);
                    myWatch2.Reset();
                    myWatch2.Start();

                    //Now build the dimension list
                    if (_list.Count >= THREAD_THRESHOLD && _coreCount >= THREAD_CORE_COUNT)
                    {
                        //Threaded
                        var threadList = new List<System.Threading.Thread>();
                        var dimensionDefList = _repositoryDefinition.DimensionList.ToList();
                        for (var ii = 0; ii < dimensionDefList.Count; ii++)
                        {
                            var dimension = _dimensionList[ii];
                            if (dimension.RefinementList.Count > 0)
                            {
                                var threader = new QueryDimensionTheader(newResults, queriedListRealized, _dimensionList, _repositoryDefinition, ii);
                                var checkDimension = _dimensionList.First(y => y.Name == dimensionDefList[ii].Name);
                                var checkRefinementList = checkDimension.RefinementList.Select(z => z.DVIdx).ToList();
                                if (!newResults.Query.DimensionValueList.Any(x => checkRefinementList.Contains(x)))
                                {
                                    var t = new System.Threading.Thread(threader.ProcessDimension);
                                    threadList.Add(t);
                                    t.Start();
                                }
                            }
                            else
                            {
                                newResults.DimensionList.Add(new DimensionItem() { Name = dimension.Name, DIdx = dimension.DIdx, NumericBreak = dimension.NumericBreak });
                            }

                        }
                        foreach (var t in threadList) t.Join();
                    }
                    else
                    {
                        //Non-Threaded
                        for (var ii = 0; ii < _repositoryDefinition.DimensionList.Count(); ii++)
                        {
                            var dimension = _dimensionList[ii];
                            if (dimension.RefinementList.Count > 0)
                            {
                                myWatch2.Stop();
                                myWatch2.Reset();
                                myWatch2.Start();

                                var threader = new QueryDimensionTheader(newResults, queriedListRealized, _dimensionList, _repositoryDefinition, ii);
                                threader.ProcessDimension();

                                myWatch2.Stop();
                                _timeList.Add(dimension.Name + " " + myWatch2.ElapsedMilliseconds);
                            }
                            else
                            {
                                newResults.DimensionList.Add(new DimensionItem() { Name = dimension.Name, DIdx = dimension.DIdx, NumericBreak = dimension.NumericBreak });
                            }
                        }
                    }

                    #region Aggregate dimension counts

                    if (query.IncludeRefinementAgg)
                    {
                        newResults.DimensionList.ToList().ForEach(x => x.RefinementList.ForEach(z => z.AggCount = 0));
                        foreach (var item in queriedListRealized)
                        {
                            foreach (var dvidx in item.DimensionValueArray)
                            {
                                var d1 = _dimensionList.GetDimensionByDVIdx(dvidx);
                                var d2 = newResults.DimensionList.FirstOrDefault(x => x.DIdx == d1.DIdx);
                                var r = d2.RefinementList.FirstOrDefault(x => x.DVIdx == dvidx);
                                if (r != null)
                                    r.AggCount++;
                                else
                                    System.Diagnostics.Debug.WriteLine("");
                            }
                        }
                    }

                    #endregion

                    //Get dimensions with parent
                    var isMasterResults = (query.NonParsedFieldList["masterresults"] == "true" || query.NonParsedFieldList["masterresults"] == "1");
                    var defItemsWithParent = _repositoryDefinition.DimensionList.Where(y => !string.IsNullOrEmpty(y.Parent)).ToList();
                    if (!isMasterResults)
                    {
                        var isSet = true;
                        while (isSet)
                        {
                            isSet = false;
                            var childDimensions = newResults.DimensionList.Where(x => defItemsWithParent.Any(z => z.Name == x.Name)).ToList();
                            foreach (var dItem in childDimensions)
                            {
                                var childDef = defItemsWithParent.First(x => x.Name == dItem.Name);
                                var parent = newResults.DimensionList.FirstOrDefault(x => x.Name == childDef.Parent);
                                if (parent != null && parent.RefinementList.Count > 1)
                                {
                                    newResults.DimensionList.Remove(dItem);
                                    isSet = true;
                                }
                            }
                        }
                    }

                    //Now associate the parent with each dimension
                    foreach (var dimension in defItemsWithParent)
                    {
                        var d = newResults.DimensionList.FirstOrDefault(x => x.Name == dimension.Name);
                        if (d != null)
                        {
                            //This is the returned dimension object
                            //If its parent dimension is in the returned set then associate them
                            d.Parent = newResults.DimensionList.FirstOrDefault(x => x.Name == dimension.Parent);
                        }
                    }

                    myWatch2.Stop();
                    _timeList.Add(string.Empty + myWatch2.ElapsedMilliseconds);

                    var elapsed = timer.Stop();
                    newResults.ComputeTime = elapsed;
                    AddProfileItem(ActionConstants.Query, timer, query.ToString(), newResults.TotalRecordCount);
                    _system.LogRepositoryPerf(new RepositorySummmaryStats
                    {
                        ActionType = RepositoryActionConstants.Query,
                        Elapsed = timer.Elapsed,
                        RepositoryId = _repositoryDefinition.ID,
                        QueryCount = 1,
                        //QueryCount = newResults.RecordList.Count,
                    });

                    //Cache this object if necessary
                    if (_cacheControl.ShouldCache(query))
                        this.PerformCaching(qHash, newResults);

                    //Log this query
                    var worker = new LogActionDelegate(_queryLogger.Log);
                    worker.BeginInvoke(query, (int)newResults.ComputeTime, newResults.TotalRecordCount, false, null, null);

                    newResults.VersionHash = _versionHash;
                    return newResults;
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    throw;
                }
            }
        }

        #endregion

        #region Reset

        public void Clear(UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    //If loaded then clear the data array
                    if (this.IsLoaded)
                        _list.Clear();

                    var timer = new CQTimer();
                    Console.WriteLine("Clear: " + _repositoryDefinition.ID);

                    //Delete the file off disk
                    var f = Path.Combine(_cacheFolder, "repository.data");
                    if (File.Exists(f))
                        File.Delete(f);

                    AddProfileItem(ActionConstants.Clear, timer);

                    ReloadMe(_startup);
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    throw;
                }
            }
        }

        /// <summary>
        /// Reloads the repository if need be
        /// </summary>
        public void Reset(Celeriq.Server.Interfaces.ServiceStartup startup)
        {
            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    if (this.IsLoaded)
                        _list.Clear();

                    if (Directory.Exists(_cacheFolder))
                    {
                        var files = Directory.GetFiles(_cacheFolder, "*.data");
                        foreach (var f in files) File.Delete(f);
                    }

                    ReloadMe(startup);

                    Logger.LogInfo("Repository Reset: " + startup.RepositoryDefinition.ID.ToString());
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    throw;
                }
            }
        }

        //internal void ShutDown()
        //{
        //	_list.Clear();
        //	_list = null;
        //	_dimensionList.Clear();
        //	_dimensionList = null;

        //	_dDimensionCache = null;

        //	_pkList.Clear();
        //	_pkList = null;
        //	_deletedCache = null;

        //	_deletedList.Clear();
        //	_deletedList = null;
        //	_resultsCache = null;
        //	_fileCache = null;
        //	_cacheControl = null;

        //	_lastInserted = null;

        //	Logger.LogDebug("System.Core ShutDown");
        //}

        public bool ExportSchema(Celeriq.Server.Interfaces.ServiceStartup startup, UserCredentials credentials, string backupFile)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    //Copy all data files to temp folder
                    //var fi = new FileInfo(Path.Combine(startup.RepositoryFolder, _repositoryDefinition.ID.ToString() + ".celeriq"));
                    var fi = new FileInfo(Path.Combine(startup.RepositoryFolder, startup.RepositoryDefinition.ID.ToString() + ".celeriq"));
                    if (fi.Exists)
                    {
                        if (File.Exists(backupFile))
                        {
                            File.Delete(backupFile);
                            System.Threading.Thread.Sleep(400);
                        }
                        fi.CopyTo(backupFile);

                        Logger.LogInfo("Repository ExportSchema: " + startup.RepositoryDefinition.ID.ToString());
                    }
                    return true;
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        /// <summary>
        /// Copies the entire repository to an archive file
        /// </summary>
        public bool Backup(Celeriq.Server.Interfaces.ServiceStartup startup, UserCredentials credentials, string backupFile)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    var timer = new CQTimer();

                    //Ensure filename only
                    backupFile = (new FileInfo(backupFile)).Name;

                    //Create a temp folder
                    var tempFolder = Path.GetTempPath();
                    tempFolder = Path.Combine(tempFolder, Guid.NewGuid().ToString());
                    Directory.CreateDirectory(tempFolder);

                    //Get the repository folder
                    var repositoryFolder = Path.Combine(startup.RepositoryFolder, startup.RepositoryDefinition.ID.ToString());

                    //Copy all data files to temp folder
                    var files = Directory.GetFiles(repositoryFolder, "*.data");
                    foreach (var file in files)
                    {
                        var fi = new FileInfo(file);
                        File.Copy(file, Path.Combine(tempFolder, fi.Name));
                    }

                    var backupFolder = Path.Combine(startup.RepositoryFolder, "..", "Backup");
                    if (!Directory.Exists(backupFolder)) Directory.CreateDirectory(backupFolder);

                    var fullBackup = Path.Combine(backupFolder, backupFile);
                    var backupfi = new FileInfo(fullBackup);
                    if (string.IsNullOrEmpty(backupfi.Extension))
                        fullBackup = ".cqbak";

                    //Write the repository definition
                    startup.RepositoryDefinition.ToDisk(Path.Combine(tempFolder, "definition.xml"));

                    ArchiveDomain.CreateArchive(tempFolder, "*.*", fullBackup);

                    //Remove the temp folder
                    Directory.Delete(tempFolder, true);

                    AddProfileItem(ActionConstants.Backup, timer);

                    Logger.LogInfo("Repository Backup: " + startup.RepositoryDefinition.ID.ToString());

                    return true;
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        /// <summary>
        /// Given an repository archive file, this method will create a new repository from it
        /// </summary>
        public bool Restore(Celeriq.Server.Interfaces.ServiceStartup startup, UserCredentials credentials, string backupFile)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    var timer = new CQTimer();

                    //Reset the entire repository
                    this.Reset(startup);

                    var backupFolder = Path.Combine(startup.RepositoryFolder, @"..\Backup");
                    var fullBackup = Path.Combine(backupFolder, backupFile);

                    //Unzip all files
                    ArchiveDomain.ExtractArchive(_cacheFolder, fullBackup);

                    var name = _repositoryDefinition.Name;
                    var id = _repositoryDefinition.ID;

                    var defFile = Path.Combine(_cacheFolder, "definition.xml");
                    _repositoryDefinition.LoadXml(File.ReadAllText(defFile));

                    //Reset repository properties
                    _repositoryDefinition.Name = name;
                    _repositoryDefinition.ID = id;

                    //Save new repository definition
                    defFile = Path.Combine(startup.RepositoryFolder, _repositoryDefinition.ID.ToString() + ".celeriq");
                    _repositoryDefinition.ToDisk(defFile);

                    AddProfileItem(ActionConstants.Restore, timer);

                    Logger.LogInfo("Repository Restore: " + startup.RepositoryDefinition.ID.ToString());

                    //Now refresh the entire repository
                    ReloadMe(startup);

                    return true;
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        #endregion

        #region Compress

        /// <summary>
        /// The disk data for this repository will be compressed
        /// </summary>
        /// <remarks>The repository will be offline while this operation runs</remarks>
        public void Compress(UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            this.Compress();
        }

        private void Compress()
        {
            try
            {
                if (!this.IsLoaded)
                {
                    ReloadMe(_startup);
                }

                var timer = new Stopwatch();
                timer.Start();

                //If there were any deleted items then compact the file and reset the deleted items
                if (_deletedList.Count > 0)
                {
                    var timerInner = new CQTimer();

                    this.SaveData();

                    //Clear cache
                    if (_resultsCache != null)
                        _resultsCache.Clear();

                    AddProfileItem(ActionConstants.Compress, timerInner);
                    Logger.LogInfo("Repository Compress: " + _repositoryDefinition.ID.ToString());
                }

                timer.Stop();
                _system.LogRepositoryPerf(new RepositorySummmaryStats
                {
                    ActionType = RepositoryActionConstants.Compress,
                    Elapsed = (int)timer.ElapsedMilliseconds,
                    RepositoryId = _repositoryDefinition.ID,
                    QueryCount = 1,
                    //QueryCount = _list.Count,
                });
            }
            catch (Exception ex)
            {
                throw;
            }
        }

        #endregion

        #region ShutDown

        public void ShutDown()
        {
            if (!_rebuildCache) return;

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    WriteCacheFile();
                    Logger.LogInfo("Repository ShutDown: " + _repositoryDefinition.ID.ToString());
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        #endregion

        #region GetDataDiskSize

        /// <summary>
        /// Returns the size on disk of the repository data
        /// </summary>
        /// <returns></returns>
        public long GetDataDiskSize(UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            if (!Directory.Exists(_cacheFolder)) return 0;
            var files = Directory.GetFiles(_cacheFolder, "*.data");
            return files.Select(f => new FileInfo(f)).Select(fi => fi.Length).Sum();
        }

        #endregion

        #region GetDataMemorySize

        /// <summary>
        /// Returns the size of memory of the repository data
        /// </summary>
        /// <returns></returns>
        public long GetDataMemorySize(UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            return _memorySize;
        }

        #endregion

        #region GetItemCount

        /// <summary>
        /// Returns the number of items in the repository
        /// </summary>
        /// <returns></returns>
        public long GetItemCount(UserCredentials credentials)
        {
            if (!IsValidCredentials(credentials))
                throw new Exception("Invalid credentials");

            return GetItemCount();
        }

        internal long GetItemCount()
        {
            try
            {
                if (this.IsLoaded)
                    return _list.Count;
                else
                    return GetItemCountFromCache();
            }
            catch (Exception ex)
            {
                throw;
            }
        }

        #endregion

        #region GetProfile

        public ProfileItem[] GetProfile(UserCredentials credentials, long lastProfileId)
        {
            lock (_profileList)
            {
                try
                {
                    var retval = new List<ProfileItem>();
                    var index = _profileList.Select(x => x.ProfileId).ToList().IndexOf(lastProfileId);
                    if (index != -1)
                    {
                        retval.AddRange(_profileList.Skip(index + 1).ToList());
                    }
                    else
                    {
                        retval.AddRange(_profileList);
                    }
                    return retval.ToArray();
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        #endregion

        #region Private Helpers

        private static Dictionary<string, string> _credCache = new Dictionary<string, string>();

        public bool IsValidCredentials(UserCredentials credentials)
        {
            lock (_credCache)
            {
                try
                {
                    if (credentials == null)
                    {
                        Logger.LogError("Credentials was null");
                        return false;
                    }

                    var user = UserDomain.UserList.FirstOrDefault(x => x.UserName == credentials.UserName);
                    if (user == null)
                    {
                        Logger.LogError("Credentials user was not found");
                        return false;
                    }

                    if (_credCache.ContainsKey(credentials.Password))
                        return (_credCache[credentials.Password] == user.Password);

                    var decrypt = Celeriq.Utilities.SecurityHelper.Decrypt(_masterKeys.PrivateKey, credentials.Password);
                    _credCache.Add(credentials.Password, decrypt);

                    if (_credCache.Keys.Count > 200)
                        _credCache.Remove(_credCache.Keys.First());

                    return (decrypt == user.Password);
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex);
                    return false;
                }
            }
        }

        private void LoadDeletedList()
        {
            try
            {
                var fh = new FileCacheHelper<long>();
                var tlist = fh.LoadAll(_deletedCache.CacheFileName);
                tlist.ForEach(x => _deletedList.Add(x));
            }
            catch (Exception ex)
            {
                Logger.LogError(ex);
                throw;
            }
        }

        private void InitializeDimension(DimensionItem dimension, FileCacheHelper<RefinementItem> cacheFile)
        {
            try
            {
                var fh = new FileCacheHelper<RefinementItem>();
                dimension.RefinementList.AddRange(fh.LoadAll(cacheFile.CacheFileName));
            }
            catch (Exception ex)
            {
                Logger.LogError(ex);
                throw;
            }
        }

        private void LoadData()
        {
            var timer = new Stopwatch();
            timer.Start();

            try
            {
                var tlist = new List<DataItemExtension>();
                var fh = new FileCacheHelper<DataItemExtension>();
                tlist.AddRange(fh.LoadAll(_fileCache.CacheFileName));
                foreach (var item in tlist)
                {
                    if (!_deletedList.Contains(item.__RecordIndex))
                    {
                        //pre-cache the dimension values to items
                        item.DimensionValueArray.ForEach(x => _dimensionMappedItemCache[x].Add(item));
                        _list.Add(item);
                        _pkList.Add((int)item.ItemArray[_pkindex]);
                    }
                }

                timer.Stop();
                this.Compress();
                timer.Start();

                if (_list.Count != 0)
                    _maxRecordIndex = _list.Max(x => x.__RecordIndex);
                _resultsCache = new SequencedHashTable<int, DataQueryResults>();
                _isLoaded = true;
                WriteCacheFile();
                _memoryDirty = false;
            }
            catch (Exception ex)
            {
                throw;
            }

            timer.Stop();
            _system.LogRepositoryPerf(new RepositorySummmaryStats
                                      {
                                          ActionType = RepositoryActionConstants.Load,
                                          Elapsed = (int)timer.ElapsedMilliseconds,
                                          RepositoryId = _repositoryDefinition.ID,
                                          QueryCount = 1,
                                          //QueryCount = _list.Count,
                                      });
        }

        private void SaveData()
        {
            if (!_memoryDirty) return;
            try
            {
                var newCache = new FileCacheHelper<DataItem>();
                var index = 0;
                while (index < _list.Count)
                {
                    var takeCount = 1000;
                    if (index + takeCount > _list.Count)
                        takeCount = _list.Count - index;
                    newCache.WriteItem(_list.AsParallel().Skip(index).Take(takeCount).ToArray());
                    index += takeCount;
                }

                if (File.Exists(_fileCache.CacheFileName))
                    File.Delete(_fileCache.CacheFileName);

                if (File.Exists(newCache.CacheFileName))
                    File.Move(newCache.CacheFileName, _fileCache.CacheFileName);

                _deletedList.Clear();
                if (File.Exists(_deletedCache.CacheFileName))
                    File.Delete(_deletedCache.CacheFileName);

                _memoryDirty = false;
                Logger.LogInfo("Repository SaveData: " + _repositoryDefinition.ID.ToString());
            }
            catch (Exception ex)
            {
                Logger.LogError(ex);
                throw;
            }
        }

        private void ClearCache()
        {
            try
            {
                _resultsCache = new SequencedHashTable<int, DataQueryResults>();
            }
            catch (Exception ex)
            {
                throw;
            }
        }

        /// <summary>
        /// Determines if this result set should be cached and if so do it
        /// </summary>
        private void PerformCaching(int qHash, DataQueryResults results)
        {
            if (!_resultsCache.ContainsKey(qHash))
            {
                //Remove the first item cached if we have hit cache limit
                if (_resultsCache.Count >= _cacheControl.MaxItems)
                {
                    var k = _resultsCache.OrderedKeys.FirstOrDefault();
                    if (k != null) _resultsCache.Remove(k);
                    else
                    {
                        System.Diagnostics.Debug.WriteLine("Key not found. 0x2998");
                        Logger.LogDebug("PerformCaching has no OrderedKeys but " + _resultsCache.Count + " cached items");
                    }
                }
                _resultsCache.Add(qHash, results);
            }
            else
            {
                _resultsCache[qHash] = results;
            }
        }

        private void ProcessDimensions(DataItemExtension newItem)
        {
            try
            {
                var dimensionDefList = _repositoryDefinition.DimensionList.ToList();
                for (var ii = 0; ii < dimensionDefList.Count; ii++)
                {
                    var dimensionDef = dimensionDefList[ii];
                    var dimension = _dimensionList.First(x => x.Name == dimensionDef.Name);
                    var field = _repositoryDefinition.FieldList.First(x => x.Name == dimensionDef.Name);
                    var fieldIndex = _repositoryDefinition.FieldList.IndexOf(field);

                    #region String

                    if (field.DataType == RepositoryDefinition.DataTypeConstants.String)
                    {
                        if (!string.IsNullOrEmpty((string)newItem.ItemArray[fieldIndex]))
                        {
                            lock (dimension)
                            {
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.FieldValue == (string)newItem.ItemArray[fieldIndex]);
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem() { Count = 1, DVIdx = dimension.GetNextDVIdx(), FieldValue = (string)newItem.ItemArray[fieldIndex] };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }

                    #endregion

                    #region List

                    else if (field.DataType == RepositoryDefinition.DataTypeConstants.List)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValueList = (string[])newItem.ItemArray[fieldIndex];
                                foreach (var fieldValue in fieldValueList)
                                {
                                    var refinement = dimension.RefinementList.FirstOrDefault(x => x.FieldValue == fieldValue);
                                    if (refinement == null)
                                    {
                                        refinement = new RefinementItem() { Count = 1, DVIdx = dimension.GetNextDVIdx(), FieldValue = fieldValue };
                                        dimension.RefinementList.Add(refinement);
                                        _dDimensionCache[ii].WriteItem(refinement);
                                        _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                    }
                                    else
                                    {
                                        refinement.Count++;
                                    }

                                    //This is for the non-list dimensions for fast dimensions aggregate computation
                                    newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                    //Add the list of all dimension values for this item
                                    if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                        newItem.DimensionValueArray.Add(refinement.DVIdx);

                                }
                            }
                        }
                    }

                    #endregion

                    #region Integer

                    else if ((field.DataType == RepositoryDefinition.DataTypeConstants.Int) && dimension.NumericBreak == null)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValue = (int)newItem.ItemArray[fieldIndex];
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.FieldValue == fieldValue.ToString());
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem()
                                                     {
                                                         Count = 1,
                                                         DVIdx =
                                                             dimension.GetNextDVIdx(),
                                                         FieldValue = fieldValue.ToString(),
                                                         MinValue = fieldValue,
                                                         MaxValue = fieldValue
                                                     };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }
                    else if ((field.DataType == RepositoryDefinition.DataTypeConstants.Int) && dimension.NumericBreak != null)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValue = (int)newItem.ItemArray[fieldIndex];
                                var minLevel = ((long)fieldValue / dimension.NumericBreak.Value) * dimension.NumericBreak.Value;
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.MinValue == minLevel);
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem()
                                                     {
                                                         Count = 1,
                                                         DVIdx = dimension.GetNextDVIdx(),
                                                         FieldValue = minLevel.ToString("###,###,###,##0") + " - " + (minLevel + dimension.NumericBreak.Value).ToString("###,###,###,##0"),
                                                         MinValue = minLevel,
                                                         MaxValue = minLevel + dimension.NumericBreak.Value
                                                     };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }

                    #endregion

                    #region Bool

                    else if (field.DataType == RepositoryDefinition.DataTypeConstants.Bool)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValue = (bool)newItem.ItemArray[fieldIndex];
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.FieldValue == fieldValue.ToString());
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem()
                                                     {
                                                         Count = 1,
                                                         DVIdx = dimension.GetNextDVIdx(),
                                                         FieldValue = fieldValue.ToString(),
                                                     };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }
                    else if ((field.DataType == RepositoryDefinition.DataTypeConstants.Int) && dimension.NumericBreak != null)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValue = (int)newItem.ItemArray[fieldIndex];
                                var minLevel = ((long)fieldValue / dimension.NumericBreak.Value) * dimension.NumericBreak.Value;
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.MinValue == minLevel);
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem()
                                                     {
                                                         Count = 1,
                                                         DVIdx = dimension.GetNextDVIdx(),
                                                         FieldValue = minLevel.ToString("###,###,###,##0") + " - " + (minLevel + dimension.NumericBreak.Value).ToString("###,###,###,##0"),
                                                         MinValue = minLevel,
                                                         MaxValue = minLevel + dimension.NumericBreak.Value
                                                     };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }

                    #endregion

                    #region DateTime

                    else if (field.DataType == RepositoryDefinition.DataTypeConstants.DateTime)
                    {
                        if (newItem.ItemArray[fieldIndex] != null)
                        {
                            lock (dimension)
                            {
                                var fieldValue = (DateTime)newItem.ItemArray[fieldIndex];
                                var refinement = dimension.RefinementList.FirstOrDefault(x => x.FieldValue == fieldValue.ToString(DimensionItem.DateTimeFormat));
                                if (refinement == null)
                                {
                                    refinement = new RefinementItem()
                                                     {
                                                         Count = 1,
                                                         DVIdx = dimension.GetNextDVIdx(),
                                                         FieldValue = fieldValue.ToString(DimensionItem.DateTimeFormat),
                                                     };
                                    dimension.RefinementList.Add(refinement);
                                    _dDimensionCache[ii].WriteItem(refinement);
                                    _dimensionMappedItemCache.Add(refinement.DVIdx, new List<DataItemExtension>());
                                }
                                else
                                {
                                    refinement.Count++;
                                }

                                //This is for the non-list dimensions for fast dimensions aggregate computation
                                newItem.DimensionSingularValueArray[ii] = refinement.DVIdx;

                                //Add the list of all dimension values for this item
                                if (!newItem.DimensionValueArray.Contains(refinement.DVIdx))
                                    newItem.DimensionValueArray.Add(refinement.DVIdx);

                            }
                        }
                    }

                    #endregion

                    else if (field.DataType == RepositoryDefinition.DataTypeConstants.Float)
                    {
                        throw new Exception("Unsupported dimension data type! Type '" + field.DataType.ToString() + "'");
                    }

                    else if (field.DataType == RepositoryDefinition.DataTypeConstants.GeoCode)
                    {
                        throw new Exception("Unsupported dimension data type! Type '" + field.DataType.ToString() + "'");
                    }

                    else
                    {
                        throw new Exception("Unsupported dimension data type! Type '" + field.DataType.ToString() + "'");
                    }

                }

                //Update the dimension cache
                foreach (var dvidx in newItem.DimensionValueArray)
                {
                    _dimensionMappedItemCache[dvidx].Add(newItem);
                }

            }
            catch (Exception ex)
            {
                Logger.LogError(ex);
                throw;
            }
        }

        private void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            //Ensure we must rebuild and has been more than 10 seconds since change
            if (!_rebuildCache) return;
            if (DateTime.Now.Subtract(_lastChange).TotalSeconds < 10) return;

            using (var q = new AcquireWriterLock(_locker))
            {
                try
                {
                    WriteCacheFile();
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        private bool IsItemValid(DataItemExtension item)
        {
            if (item.ItemArray == null)
            {
                Logger.LogWarning("IsItemValid: ItemArray Null");
                return false;
            }
            if (item.ItemArray.Length != _repositoryDefinition.FieldList.Count)
            {
                Logger.LogWarning("IsItemValid: ItemArray Count mismatch: " + item.ItemArray.Length + "/" + _repositoryDefinition.FieldList.Count);
                return false;
            }

            var index = 0;
            foreach (var field in _repositoryDefinition.FieldList)
            {
                if (item.ItemArray[index] != null)
                {
                    switch (field.DataType)
                    {
                        case RepositoryDefinition.DataTypeConstants.Bool:
                            if (!(item.ItemArray[index] is bool)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.DateTime:
                            if (!(item.ItemArray[index] is DateTime)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.Float:
                            if (!(item.ItemArray[index] is double)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.GeoCode:
                            if (!(item.ItemArray[index] is GeoCode)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.Int:
                            if (!(item.ItemArray[index] is int)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.String:
                            if (!(item.ItemArray[index] is string)) return false;
                            break;
                        case RepositoryDefinition.DataTypeConstants.List:
                            if (!(item.ItemArray[index] is string[])) return false;
                            break;
                        default:
                            Logger.LogWarning("IsItemValid: Unknown data type: " + field.DataType.ToString());
                            throw new Exception("Unknown data type!");
                    }
                }
                index++;
            }
            return true;
        }

        private System.Linq.Expressions.Expression GetEqualsExpr(System.Linq.Expressions.ParameterExpression param, string property, string value)
        {
            var prop = System.Linq.Expressions.Expression.Property(param, property);
            var val = System.Linq.Expressions.Expression.Constant(value);
            return System.Linq.Expressions.Expression.Equal(prop, val);
        }

        private void AddProfileItem(ActionConstants action, CQTimer timer)
        {
            AddProfileItem(action, timer, null, 0);
        }

        private void AddProfileItem(ActionConstants action, CQTimer timer, string query, int itemsAffected)
        {
            lock (_profileList)
            {
                try
                {
                    long profileId = 1;
                    if (_profileList.Count > 0) profileId = _profileList.Select(x => x.ProfileId).Max() + 1;

                    var newItem = new ProfileItem()
                                      {
                                          Action = action,
                                          Duration = timer.Elapsed,
                                          Name = _repositoryDefinition.Name,
                                          ProfileId = profileId,
                                          RepositoryId = _repositoryDefinition.ID,
                                          StartTime = timer.StartTime,
                                          Query = query,
                                          ItemsAffected = itemsAffected,
                                      };
                    _profileList.Add(newItem);

                    //Remove first item until we are back at defined max
                    while (_profileList.Count > MAX_PROFILE_COUNT)
                    {
                        _profileList.RemoveAt(0);
                    }
                }
                catch (Exception ex)
                {
                    throw;
                }
            }
        }

        #endregion
    }

}